#需求：爬取梨视频的视频数据
import requests
from lxml import etree
from multiprocessing.dummy import Pool
import os
headers={
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36 Edg/92.0.902.67'
}
#原则：线程池处理的是阻塞且耗时的操作
if not os.path.exists('./梨视频'):
    os.mkdir('./梨视频')
#对下述url发起请求解析出视频详情页的URL和视频名称
url='https://www.pearvideo.com/category_8'
page_text = requests.get(url=url,headers=headers).text

tree=etree.HTML(page_text)
urls=[]
li_list = tree.xpath('//ul[@id="listvideoListUl"]/li')
for li in li_list:
    name=li.xpath('./div/a/div[2]/text()')[0]+'.mp4'
    href=li.xpath('./div/a/@href')[0]
    #print(href)
    videoId=href.split('_')[1]
    #print(videoId)
    headers={
        'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36 Edg/92.0.902.67',
        'Referer':'https://www.pearvideo.com/'+href,
    }
    videoHref='https://www.pearvideo.com/videoStatus.jsp?contId='+videoId
    videoUrl=requests.get(url=videoHref,headers=headers).json()
    #print(videoUrl)
    video=videoUrl["videoInfo"]["videos"]["srcUrl"]
    video=video.replace(video.split('/')[-1].split('-')[0],'cont-%s'%videoId)
    dic={
        'name':name,
        'url':video
    }
    urls.append(dic)
# print(urls)

def getVideoData(dic):
    url=dic['url']
    videoData = requests.get(url=url,headers=headers).content
    with open('./梨视频/'+dic['name'],'wb') as fp:
        fp.write(videoData)
        print(dic['name'],'下载成功')

pool = Pool(4)
pool.map(getVideoData,urls)
